import pandas as pd
import json
import re
import numpy as np
import os
import plotly.io as pio
pio.renderers
pio.renderers.default = 'notebook'
Results were collected between 11.06. -- 23.07.2021
Prior to parsing, spurious responses were discarded on the basis of:
# Loading Data
RESULTS_FILEPATH = "data/20210726170044_results.json"
figure_save_dir = os.path.splitext(RESULTS_FILEPATH)[0]
ids_to_exclude = [
"1624013360083", #inappropriate comment, extreme values
"1623688924533", #test input
]
with open(RESULTS_FILEPATH) as f:
# remove image0/image1 vars since it prevents proper df merging
text = f.read()
text = re.sub(r"_image[0-3]", "", text)
text = re.sub(r"user_profiling_", "user_", text)
text = re.sub(r"user_", "userProfiling_", text)
lines = text.splitlines()
data = []
for line in lines[1:-1]:
entry = json.loads(line)
entry["id"] = str(round(entry["id"]))
if "userProfiling_aiFamiliarity" not in entry['userProfiling_'].keys():
entry["userProfiling_"]["userProfiling_aiFamiliarity"]="missing"
data.append(entry)
df = pd.json_normalize(data)
df.drop(labels=["__v", "_id.$oid"], axis=1, inplace=True)
renamed = [name.split("_.")[-1] for name in df.columns]
col_rename = {i: j for i, j in zip(df.columns, renamed)}
df = df.rename(columns=col_rename)
df.drop_duplicates(subset=['id'], inplace=True)
df = df[~df.id.isin(ids_to_exclude)]
df = df[df.userProfiling_position != "other"]
df.set_index("id", inplace=True)
df
| userProfiling_age | userProfiling_position | userProfiling_useOfAI | userProfiling_useOfDP | userProfiling_mlFamiliarity | userProfiling_aiFamiliarity | saliencyMaps_globalSaliency_understandability | saliencyMaps_globalSaliency_usability | saliencyMaps_globalSaliency_informativeness | saliencyMaps_globalSaliency_value | ... | trustScores_borderlineCases_value | userProfiling_useOfAI_details | saliencyMaps_globalSaliency_comments | saliencyMaps_localSaliency_comments | conceptAttribution_textAttributes_comments | trustScores_borderlineCases_comments | userProfiling_comments | counterfactuals_twoAxisCounterfactuals_comments | counterfactuals_prototypeInterpolation_comments | userProfiling_position-Comment | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| id | |||||||||||||||||||||
| 1623319499818 | 30-40 | Assisting physician (Assistenzarzt) for pathology/neuropathology | in routine diagnostics | in routine diagnostics | 1 | missing | 5 | 5 | 6 | 6 | ... | 6 | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 1623330321488 | 41-50 | Researcher in pathology/neuropathology | in research | in research | 3 | missing | 7 | 7 | 7 | 7 | ... | 3 | from MindPeak (BreastIHC), from VMscope (Cognition Master) | There is one cell slightly to the right of the center that is labeled as positive and is quite darkly stained by IHC, but does not show up on the saliency map, which is a bit surprising. It looks like an endothelial cell. | Per-cell saliency is probably too much detail. | This is excellent but I think there could be quite a lot of additional factors, so this would need some supervision. | High confidence should provide examples of both classes (pos. and neg.), no? | NaN | NaN | NaN | NaN |
| 1623391402856 | 30-40 | Researcher in pathology/neuropathology | [in research] | [in research] | 7 | 5 | 6 | 3 | 4 | 5 | ... | 6 | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 1623391916479 | 41-50 | Technician (MTA) for pathology/neuropathology | [in routine diagnostics] | [in research] | 7 | 4 | 5 | 7 | 3 | 6 | ... | 6 | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 1623439901422 | 30-40 | Trainee (Assistenzarzt) in pathology/neuropathology | [in research] | [in research] | 2 | 5 | 4 | 5 | 6 | 4 | ... | 5 | QuPath immunohistochemistry positive cell detection | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 1623441394465 | 30-40 | Consultant (Facharzt) for pathology/neuropathology | [in research] | [in research] | 7 | 7 | 6 | 5 | 5 | 6 | ... | 5 | Aiforia | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 1623655000204 | 51-60 | Researcher in pathology/neuropathology | [none] | [none] | 2 | 3 | 5 | 5 | 5 | 5 | ... | 5 | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 1623674498302 | 41-50 | Consultant (Facharzt) for pathology/neuropathology | [none] | [in routine diagnostics] | 4 | 5 | 1 | 1 | 1 | 1 | ... | 2 | NaN | NaN | NaN | NaN | Confidence for what? positive or negative or both? | NaN | NaN | NaN | NaN |
| 1623676868025 | 41-50 | Consultant (Facharzt) for pathology/neuropathology | [none] | [none] | 5 | 5 | 3 | 3 | 2 | 2 | ... | 6 | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 1623678209878 | 51-60 | Consultant (Facharzt) for pathology/neuropathology | [in routine diagnostics, in research] | [in research, in routine diagnostics] | 4 | 2 | 6 | 6 | 5 | 5 | ... | 4 | Roche Diagnostics | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 1623681765021 | 41-50 | Consultant (Facharzt) for pathology/neuropathology | [none] | [none] | 5 | 5 | 4 | 4 | 4 | 4 | ... | 4 | NaN | NaN | NaN | NaN | NaN | I was in a US based practice that was nearly 100% digital for ~ 2 years (2018-2020). We did not use any AI tools but our non-cytology / non-bone marrow specimens were 100% digitized for clinical sign out (Philips system). Based on clinical expectations for turn around time AND the significant decrease in reimbursement for pathology in the US—digital pathology IS NOT finically viable in the US. Profit margins are not there for ROI based on the cost of scanners and even the process of having histology slides put on the scanner in our histology lab led to significant problems with our process. Until the steps of going from tissue to an imaging are automated and streamlined there will be significant barriers to digital pathology (first step necessary for AI to even be in the conversation) and AI in clinical practice on the US. Just FYI. | NaN | NaN | NaN |
| 1623682375207 | 30-40 | Consultant (Facharzt) for pathology/neuropathology | [in routine diagnostics, in research] | [in routine diagnostics, in research] | 2 | 4 | 4 | 4 | 5 | 3 | ... | 5 | tumor cellularity count; pattern recognition; both made in-house | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 1623721801858 | 30-40 | Researcher in pathology/neuropathology | [in research] | [in research] | 7 | 7 | 5 | 3 | 3 | 4 | ... | 3 | NaN | NaN | NaN | NaN | I felt I needed the pos or neg labels in addition to the confidence | NaN | It seems there may be many counterfactual examples that are more closely related to the positive example which could be useful for understanding the nuances and building trust | Like previous, initial reaction is that interpolation is useful but doesn't give the full picture of possible negatives. Perhaps clicking around each cell and seeing the interpolation for each would feel more useful | NaN |
| 1623726946779 | 41-50 | Researcher in pathology/neuropathology | [in research, in routine diagnostics] | [in research, in routine diagnostics] | 7 | 7 | 2 | 5 | 6 | 2 | ... | 5 | Paige | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 1623743656931 | 41-50 | Technician (MTA) for pathology/neuropathology | [in research] | [in research] | 7 | 6 | 7 | 7 | 7 | 7 | ... | 7 | Lrp | NaN | NaN | NaN | NaN | NaN | Do not understand this question | What is a counterfactual? | NaN |
| 1623964393367 | 51-60 | Researcher in pathology/neuropathology | [in research] | [in research] | 6 | 7 | 5 | 3 | 3 | 2 | ... | 5 | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 1624882031310 | 41-50 | Consultant (Fachärztin*arzt) for pathology/neuropathology | [in research] | [in research] | 6 | 4 | 5 | 6 | 6 | 6 | ... | 7 | Metastases detection in lymph nodes | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 1626010603300 | 41-50 | Technician (MTA) for pathology/neuropathology | [in routine diagnostics] | [in research] | 6 | 4 | 3 | 7 | 2 | 7 | ... | 6 | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 1626160447499 | 41-50 | Consultant (Fachärztin*arzt) for pathology/neuropathology | [none] | [in research] | 2 | 3 | 2 | 2 | 2 | 2 | ... | 3 | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 1626173121093 | 41-50 | Consultant (Fachärztin*arzt) for pathology/neuropathology | [in research] | [in research, in routine diagnostics] | 3 | 5 | 5 | 4 | 4 | 4 | ... | 7 | IHC evaluation | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 1626186219117 | 30-40 | Trainee (Assistenzärztin*arzt) in pathology/neuropathology | [none] | [in routine diagnostics, in research] | 6 | 1 | 2 | 2 | 2 | 2 | ... | 3 | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 1626187937622 | 30-40 | Consultant (Fachärztin*arzt) for pathology/neuropathology | [none] | [in research] | 2 | 1 | 3 | 3 | 3 | 3 | ... | 5 | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 1626261692711 | over 60 | Consultant (Fachärztin*arzt) for pathology/neuropathology | [none] | [none] | 6 | 7 | 6 | 7 | 6 | 5 | ... | 4 | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 1626812471740 | 30-40 | Trainee (Assistenzärztin*arzt) in pathology/neuropathology | [in research] | [in routine diagnostics, in research] | 3 | 3 | 3 | 3 | 3 | 5 | ... | 5 | Qpath, automated ki67 counting | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 1627300327909 | 41-50 | Consultant (Fachärztin*arzt) for pathology/neuropathology | [none] | [in routine diagnostics] | 2 | 2 | 1 | 1 | 1 | 1 | ... | 6 | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
25 rows × 43 columns
user_df = df[
[
"userProfiling_age",
"userProfiling_position",
"userProfiling_useOfDP",
"userProfiling_useOfAI",
"userProfiling_useOfAI_details",
"userProfiling_mlFamiliarity",
"userProfiling_aiFamiliarity"
]
]
fields = ["Understandability", "Usability", "Informativeness", "Value"]
space = " " * 4
labels = [
f"I find the explanation intuitively understandable{space}",
f"The explanation helps me to understand factors relevant to the algorithm{space}",
f"The explanation helps me to decide whether I can trust the generated annotations{space}",
f"The explanation provides me with valuable information for my work{space}",
]
instance_identifiers = {
"Counterfactuals (One-axis)": "counterfactuals_prototypeInterpolation",
"Counterfactuals (Two-axis)": "counterfactuals_twoAxisCounterfactuals",
"Saliency Map (Local)": "saliencyMaps_localSaliency",
"Saliency Map (Global)": "saliencyMaps_globalSaliency",
"Concept Attribution": "conceptAttribution_textAttributes",
"Prototypes": "prototypes_prototypes",
"Trust Scores": "trustScores_borderlineCases",
}
result_dataframes = dict()
comments = dict()
for name, id_ in instance_identifiers.items():
result_dataframes[name] = df[[f"{id_}_{field.lower()}" for field in fields]]
result_dataframes[name].columns = labels
comments_column = f"{id_}_comments"
if comments_column in df.columns:
comments[name] = df[[comments_column]].dropna().values.squeeze(axis=1).tolist()
else:
comments[name] = []
df1 = pd.DataFrame()
for name, df in result_dataframes.items():
df1[name] = df[df > 4].count()
df1 = df1.transpose()
for label in labels:
df1 = df1.sort_values(by=label, ascending=False)
index_sorted = df1.index
df1
| I find the explanation intuitively understandable | The explanation helps me to understand factors relevant to the algorithm | The explanation helps me to decide whether I can trust the generated annotations | The explanation provides me with valuable information for my work | |
|---|---|---|---|---|
| Trust Scores | 19 | 14 | 18 | 17 |
| Counterfactuals (One-axis) | 19 | 16 | 16 | 16 |
| Concept Attribution | 17 | 17 | 13 | 15 |
| Counterfactuals (Two-axis) | 15 | 14 | 14 | 14 |
| Prototypes | 20 | 14 | 13 | 12 |
| Saliency Map (Global) | 13 | 12 | 11 | 12 |
| Saliency Map (Local) | 12 | 14 | 9 | 9 |
comments
{'Counterfactuals (One-axis)': ["Like previous, initial reaction is that interpolation is useful but doesn't give the full picture of possible negatives. Perhaps clicking around each cell and seeing the interpolation for each would feel more useful",
'What is a counterfactual? '],
'Counterfactuals (Two-axis)': ['It seems there may be many counterfactual examples that are more closely related to the positive example which could be useful for understanding the nuances and building trust',
'Do not understand this question'],
'Saliency Map (Local)': ['Per-cell saliency is probably too much detail. '],
'Saliency Map (Global)': ['There is one cell slightly to the right of the center that is labeled as positive and is quite darkly stained by IHC, but does not show up on the saliency map, which is a bit surprising. It looks like an endothelial cell. '],
'Concept Attribution': ['This is excellent but I think there could be quite a lot of additional factors, so this would need some supervision. '],
'Prototypes': [],
'Trust Scores': ['High confidence should provide examples of both classes (pos. and neg.), no? ',
'Confidence for what? positive or negative or both?',
'I felt I needed the pos or neg labels in addition to the confidence']}
from typing import Optional
from collections import Counter
import plotly
import plotly.graph_objects as go
import plotly.express as px
from PIL import Image
import io
from IPython.display import display
def plotly_figure_to_image(fig, extension: str = 'png', **kwargs):
img_bytes = fig.to_image(format=extension, **kwargs)
return Image.open(io.BytesIO(img_bytes))
def save_image(image, name: str, save_dir: str = "images", extension: str="png"):
if not os.path.exists(save_dir):
os.mkdir(save_dir)
filename = "".join(i for i in name if i not in "\\/:*?<>|()- ")
filepath = f"{save_dir}/{filename}.{extension}"
image.save(filepath)
def stackedBarChartDF(
df: pd.DataFrame,
title: str,
palette: list,
diverging: bool = False,
attributes: list = ["Understandability", "Usability", "Informativeness", "Value"],
labels: Optional[list] = None,
save_fig: bool = False,
save_name: Optional[str] = None,
save_dir: str = "images",
width: int = 1200,
height: int = 400,
font: dict = {'size' : 14},
web_display: bool = False,
legend: dict=dict(
orientation="h",
yanchor="bottom",
y=-0.4,
xanchor="center",
x=0.5),
showlegend: bool=False,
**kwargs
):
counters = [Counter(df[attribute]) for attribute in attributes]
fig = go.Figure() # type: ignore
category_order = [
"Strongly<br>disagree",
"Disagree<br>",
"Slightly<br>disagree",
"Neutral<br>",
"Slightly<br>agree",
"Agree<br>",
"Strongly<br>agree",
]
number_order = [1, 2, 3, 4, 5, 6, 7]
labels = labels if labels else attributes
def add_bar_trace(num, transform, showlegend: bool = True):
rating = number_order[num]
counts = [counter[rating] for counter in counters]
total_count = sum(counters[0].values())
xvals = [transform(count) / total_count for count in counts]
fig.add_trace(
go.Bar(
x=xvals,
y=labels,
# text=counts,
# textposition='inside',
orientation="h",
name=category_order[num],
width=0.8,
marker_color=palette[num],
legendgroup=category_order[num],
showlegend=False,
) # type: ignore
)
if diverging:
# negative side
add_bar_trace(3, lambda x: x * -0.5)
for num in reversed(range(0, 3)):
add_bar_trace(num, lambda x: x * -1)
# positive side
add_bar_trace(3, lambda x: x * 0.5, showlegend=False)
for num in range(4, 7):
add_bar_trace(num, lambda x: x)
else:
for num in reversed(range(0,7)):
add_bar_trace(num, lambda x: x)
for num in reversed(range(0, 7)):
fig.add_trace(
go.Bar(
x=[0] * 7,
y=labels,
orientation="h",
name=category_order[num],
marker_color=palette[num],
legendgroup=category_order[num]
) # type: ignore
)
fig.update_layout(
barmode="relative" if diverging else "stack",
yaxis_autorange="reversed",
# title=dict(
# text=title,
# x=0.0,
# y=0.92
# ),
xaxis={"tick0": 0, "tickformat": "%", "range": [-1, 1]},
legend=legend,
showlegend=showlegend,
font=font,
autosize=False,
**kwargs
)
if not diverging:
fig.update_layout(
xaxis = {
"range" : [0, 1],
'side': 'top',
"color" : palette[-1]
},
xaxis2= {
'anchor': 'y',
'overlaying': 'x',
'side': 'bottom',
"tickformat": "%",
"range": [1, 0],
"color" : palette[0]
})
fig.data[-1].update(xaxis='x2')
im = plotly_figure_to_image(fig, width=width, height=height, validate=True)
if web_display:
fig.show()
else:
display(im)
if save_fig:
save_image(im, save_name if save_name else title, save_dir)
palette = [plotly.colors.diverging.RdBu[n] for n in [2, 3, 4, 5, 6, 7, 8]] # type: ignore
space = " " * 4
labels = [
f"I find the explanation intuitively understandable{space}",
f"The explanation helps me to understand factors relevant to the algorithm{space}",
f"The explanation helps me to decide whether I can trust the generated annotations{space}",
f"The explanation provides me with valuable information for my work{space}",
]
index=0
width = 1900
font={'size':24,'family':'Times New Roman'}
for index, name in enumerate(index_sorted[:-1]):
stackedBarChartDF(
result_dataframes[name],
name,
palette,
attributes=labels,
showlegend=False,
web_display=False,
save_fig=True,
save_dir=figure_save_dir,
save_name=f"{index}_{name}",
font=font,
width=width,
height=300,
margin=dict(
r=2,
l=0,
t=10,
b=10,
)
)
final_index = index_sorted[-1]
stackedBarChartDF(
result_dataframes[final_index],
final_index,
palette,
attributes=labels,
showlegend=True,
web_display=False,
save_fig=True,
save_dir=figure_save_dir,
save_name=f"{index+1}_{final_index}",
font=font,
width=width,
height=380,
margin=dict(
r=2,
l=0,
t=10,
b=80,
),
legend=dict(
font=font,
orientation="h",
yanchor="bottom",
y=-0.5,
xanchor="center",
x=0.5,
valign="top",
itemwidth=40),
)
for name, df in result_dataframes.items():
df.columns = labels
method='spearman'
px.imshow(
df.corr(method=method),
title=f"{method} correlation between responses for {name}",
zmin=0).show()
for comment in comments[name]:
print(comment)
Like previous, initial reaction is that interpolation is useful but doesn't give the full picture of possible negatives. Perhaps clicking around each cell and seeing the interpolation for each would feel more useful What is a counterfactual?
It seems there may be many counterfactual examples that are more closely related to the positive example which could be useful for understanding the nuances and building trust Do not understand this question
Per-cell saliency is probably too much detail.
There is one cell slightly to the right of the center that is labeled as positive and is quite darkly stained by IHC, but does not show up on the saliency map, which is a bit surprising. It looks like an endothelial cell.
This is excellent but I think there could be quite a lot of additional factors, so this would need some supervision.
High confidence should provide examples of both classes (pos. and neg.), no? Confidence for what? positive or negative or both? I felt I needed the pos or neg labels in addition to the confidence
import plotly.express as px
df2 = pd.DataFrame()
category_order = [
"Strongly<br>disagree",
"Disagree<br>",
"Slightly<br>disagree",
"Neutral<br>",
"Slightly<br>agree",
"Agree<br>",
"Strongly<br>agree",
]
number_order = [1, 2, 3, 4, 5, 6, 7]
response_dict = dict(zip(number_order, category_order))
for name in index_sorted:
df = result_dataframes[name]
df2[name] = df.median()
px.imshow(
df2,
title="median responses from 1=strongly disagree to 7=strongly agree",
zmin=1,
zmax= 7,
color_continuous_scale=palette
)
labels = [
f"I find the explanation intuitively understandable",
f"The explanation helps me to understand factors relevant to the algorithm",
f"The explanation helps me to decide whether I can trust the generated annotations",
f"The explanation provides me with valuable information for my work",
]
for name, df in result_dataframes.items():
print(name)
for index, column in enumerate(df.columns):
percentage_agree = df[df[column] > 4][column].count() / df[column].count()
percentage_stronglyagree = df[df[column] > 5][column].count() / df[column].count()
percentage_disagree = df[df[column] < 4][column].count() / df[column].count()
percentage_stronglydisagree = df[df[column] < 3][column].count() / df[column].count()
print(
'\t', labels[index], "\n\t\t",
'{:.0%}'.format(percentage_stronglydisagree), " disagree or strongly disagree", " | ",
'{:.0%}'.format(percentage_disagree), " disagree", " | ",
'{:.0%}'.format(percentage_agree), " agree", " | "
'{:.0%}'.format(percentage_stronglyagree), " agree or strongly agree")
Counterfactuals (One-axis) I find the explanation intuitively understandable 8% disagree or strongly disagree | 12% disagree | 76% agree | 60% agree or strongly agree The explanation helps me to understand factors relevant to the algorithm 16% disagree or strongly disagree | 24% disagree | 64% agree | 48% agree or strongly agree The explanation helps me to decide whether I can trust the generated annotations 16% disagree or strongly disagree | 24% disagree | 64% agree | 44% agree or strongly agree The explanation provides me with valuable information for my work 24% disagree or strongly disagree | 28% disagree | 64% agree | 28% agree or strongly agree Counterfactuals (Two-axis) I find the explanation intuitively understandable 20% disagree or strongly disagree | 28% disagree | 60% agree | 32% agree or strongly agree The explanation helps me to understand factors relevant to the algorithm 12% disagree or strongly disagree | 28% disagree | 56% agree | 36% agree or strongly agree The explanation helps me to decide whether I can trust the generated annotations 16% disagree or strongly disagree | 24% disagree | 56% agree | 32% agree or strongly agree The explanation provides me with valuable information for my work 20% disagree or strongly disagree | 28% disagree | 56% agree | 20% agree or strongly agree Saliency Map (Local) I find the explanation intuitively understandable 16% disagree or strongly disagree | 40% disagree | 48% agree | 32% agree or strongly agree The explanation helps me to understand factors relevant to the algorithm 20% disagree or strongly disagree | 28% disagree | 56% agree | 36% agree or strongly agree The explanation helps me to decide whether I can trust the generated annotations 20% disagree or strongly disagree | 32% disagree | 36% agree | 28% agree or strongly agree The explanation provides me with valuable information for my work 24% disagree or strongly disagree | 40% disagree | 36% agree | 20% agree or strongly agree Saliency Map (Global) I find the explanation intuitively understandable 20% disagree or strongly disagree | 36% disagree | 52% agree | 24% agree or strongly agree The explanation helps me to understand factors relevant to the algorithm 16% disagree or strongly disagree | 40% disagree | 48% agree | 28% agree or strongly agree The explanation helps me to decide whether I can trust the generated annotations 24% disagree or strongly disagree | 44% disagree | 44% agree | 28% agree or strongly agree The explanation provides me with valuable information for my work 28% disagree or strongly disagree | 36% disagree | 48% agree | 28% agree or strongly agree Concept Attribution I find the explanation intuitively understandable 12% disagree or strongly disagree | 28% disagree | 68% agree | 52% agree or strongly agree The explanation helps me to understand factors relevant to the algorithm 16% disagree or strongly disagree | 24% disagree | 68% agree | 48% agree or strongly agree The explanation helps me to decide whether I can trust the generated annotations 16% disagree or strongly disagree | 40% disagree | 52% agree | 40% agree or strongly agree The explanation provides me with valuable information for my work 16% disagree or strongly disagree | 28% disagree | 60% agree | 44% agree or strongly agree Prototypes I find the explanation intuitively understandable 4% disagree or strongly disagree | 4% disagree | 80% agree | 68% agree or strongly agree The explanation helps me to understand factors relevant to the algorithm 12% disagree or strongly disagree | 28% disagree | 56% agree | 40% agree or strongly agree The explanation helps me to decide whether I can trust the generated annotations 16% disagree or strongly disagree | 40% disagree | 52% agree | 44% agree or strongly agree The explanation provides me with valuable information for my work 20% disagree or strongly disagree | 36% disagree | 48% agree | 36% agree or strongly agree Trust Scores I find the explanation intuitively understandable 0% disagree or strongly disagree | 4% disagree | 76% agree | 40% agree or strongly agree The explanation helps me to understand factors relevant to the algorithm 8% disagree or strongly disagree | 20% disagree | 56% agree | 28% agree or strongly agree The explanation helps me to decide whether I can trust the generated annotations 0% disagree or strongly disagree | 8% disagree | 72% agree | 40% agree or strongly agree The explanation provides me with valuable information for my work 4% disagree or strongly disagree | 20% disagree | 68% agree | 36% agree or strongly agree
new_user_df = user_df.copy()
new_user_df.columns = [column.split("_")[1] for column in user_df.columns]
new_user_df
| age | position | useOfDP | useOfAI | useOfAI | mlFamiliarity | aiFamiliarity | |
|---|---|---|---|---|---|---|---|
| id | |||||||
| 1623319499818 | 30-40 | Assisting physician (Assistenzarzt) for pathology/neuropathology | in routine diagnostics | in routine diagnostics | NaN | 1 | missing |
| 1623330321488 | 41-50 | Researcher in pathology/neuropathology | in research | in research | from MindPeak (BreastIHC), from VMscope (Cognition Master) | 3 | missing |
| 1623391402856 | 30-40 | Researcher in pathology/neuropathology | [in research] | [in research] | NaN | 7 | 5 |
| 1623391916479 | 41-50 | Technician (MTA) for pathology/neuropathology | [in research] | [in routine diagnostics] | NaN | 7 | 4 |
| 1623439901422 | 30-40 | Trainee (Assistenzarzt) in pathology/neuropathology | [in research] | [in research] | QuPath immunohistochemistry positive cell detection | 2 | 5 |
| 1623441394465 | 30-40 | Consultant (Facharzt) for pathology/neuropathology | [in research] | [in research] | Aiforia | 7 | 7 |
| 1623655000204 | 51-60 | Researcher in pathology/neuropathology | [none] | [none] | NaN | 2 | 3 |
| 1623674498302 | 41-50 | Consultant (Facharzt) for pathology/neuropathology | [in routine diagnostics] | [none] | NaN | 4 | 5 |
| 1623676868025 | 41-50 | Consultant (Facharzt) for pathology/neuropathology | [none] | [none] | NaN | 5 | 5 |
| 1623678209878 | 51-60 | Consultant (Facharzt) for pathology/neuropathology | [in research, in routine diagnostics] | [in routine diagnostics, in research] | Roche Diagnostics | 4 | 2 |
| 1623681765021 | 41-50 | Consultant (Facharzt) for pathology/neuropathology | [none] | [none] | NaN | 5 | 5 |
| 1623682375207 | 30-40 | Consultant (Facharzt) for pathology/neuropathology | [in routine diagnostics, in research] | [in routine diagnostics, in research] | tumor cellularity count; pattern recognition; both made in-house | 2 | 4 |
| 1623721801858 | 30-40 | Researcher in pathology/neuropathology | [in research] | [in research] | NaN | 7 | 7 |
| 1623726946779 | 41-50 | Researcher in pathology/neuropathology | [in research, in routine diagnostics] | [in research, in routine diagnostics] | Paige | 7 | 7 |
| 1623743656931 | 41-50 | Technician (MTA) for pathology/neuropathology | [in research] | [in research] | Lrp | 7 | 6 |
| 1623964393367 | 51-60 | Researcher in pathology/neuropathology | [in research] | [in research] | NaN | 6 | 7 |
| 1624882031310 | 41-50 | Consultant (Fachärztin*arzt) for pathology/neuropathology | [in research] | [in research] | Metastases detection in lymph nodes | 6 | 4 |
| 1626010603300 | 41-50 | Technician (MTA) for pathology/neuropathology | [in research] | [in routine diagnostics] | NaN | 6 | 4 |
| 1626160447499 | 41-50 | Consultant (Fachärztin*arzt) for pathology/neuropathology | [in research] | [none] | NaN | 2 | 3 |
| 1626173121093 | 41-50 | Consultant (Fachärztin*arzt) for pathology/neuropathology | [in research, in routine diagnostics] | [in research] | IHC evaluation | 3 | 5 |
| 1626186219117 | 30-40 | Trainee (Assistenzärztin*arzt) in pathology/neuropathology | [in routine diagnostics, in research] | [none] | NaN | 6 | 1 |
| 1626187937622 | 30-40 | Consultant (Fachärztin*arzt) for pathology/neuropathology | [in research] | [none] | NaN | 2 | 1 |
| 1626261692711 | over 60 | Consultant (Fachärztin*arzt) for pathology/neuropathology | [none] | [none] | NaN | 6 | 7 |
| 1626812471740 | 30-40 | Trainee (Assistenzärztin*arzt) in pathology/neuropathology | [in routine diagnostics, in research] | [in research] | Qpath, automated ki67 counting | 3 | 3 |
| 1627300327909 | 41-50 | Consultant (Fachärztin*arzt) for pathology/neuropathology | [in routine diagnostics] | [none] | NaN | 2 | 2 |
pd.set_option('display.max_colwidth', None)
px.histogram(new_user_df['position'].replace(
{
"Assisting physician (Assistenzarzt) for pathology/neuropathology" : "Trainee",
"Trainee (Assistenzarzt) in pathology/neuropathology" : "Trainee",
"Trainee (Assistenzärztin*arzt) in pathology/neuropathology" : "Trainee",
"Consultant (Facharzt) for pathology/neuropathology" : "Consultant",
"Consultant (Fachärztin*arzt) for pathology/neuropathology" : "Consultant",
"Researcher in pathology/neuropathology" : "Researcher",
"Technician (MTA) for pathology/neuropathology" : "Technician"
}
)).update_xaxes(categoryorder="total descending")
px.histogram(new_user_df['age'])
print("familiar with ML: ",new_user_df[new_user_df['mlFamiliarity'] > 4].count()[0])
print("not familiar with ML: ",new_user_df[new_user_df['mlFamiliarity'] < 4].count()[0])
print("\nfamiliar with AI: ", new_user_df[2:][new_user_df[2:]['aiFamiliarity'] > 4].count()[0])
print("not familiar with AI: ", new_user_df[2:][new_user_df[2:]['aiFamiliarity'] < 4].count()[0])
print("missing: 2")
familiar with ML: 13 not familiar with ML: 10 familiar with AI: 12 not familiar with AI: 7 missing: 2
px.histogram(new_user_df['useOfAI'].iloc[:,0])
px.histogram(new_user_df['useOfDP'])
px.scatter(x= new_user_df[2:]['mlFamiliarity'], y = new_user_df[2:]['aiFamiliarity'])
mlFamiliarity = new_user_df['mlFamiliarity'][2:].astype('int32')
aiFamiliarity = new_user_df['aiFamiliarity'][2:].astype('int32')
aiFamiliarity.corr(mlFamiliarity, method="spearman")
0.5970744598756759